# -*- coding: utf-8 -*-
# @see https://github.com/yelord/baidutongji

# 运行环境依赖
# sudo apt-get install python-pip python-dev libmysqlclient-dev
# pip install requests
# pip install rsa
# pip install MySQL-python
#
import json
import requests
import math
import StringIO, gzip
import rsa
import uuid
import re
import MySQLdb
import sys
import time
from datetime import date, timedelta


LOGIN_URL = 'https://api.baidu.com/sem/common/HolmesLoginService'
API_URL = 'https://api.baidu.com/json/tongji/v1/ReportService'
UUID = str(uuid.uuid1())  # 'parkoutlets'
ACCOUNT_TYPE = '1'  # ZhanZhang:1,FengChao:2,Union:3,Columbus:4
SITE_ID = 11055589
HOST = "m.sstt.iflyvoice.com"

db_config = {
    'host': 'test',
    'port': 3306,
    'user': 'test',
    'passwd': 'test',
    'db': 'squirrel',
    'charset' : 'utf8'
}

#数据库
conn = MySQLdb.connect(**db_config)
conn.autocommit(True)
cursor = conn.cursor()

# 强制使用utf8
cursor.execute('SET NAMES utf8')
cursor.execute("SET CHARACTER SET utf8")
cursor.execute("SET character_set_connection=utf8")

# 保存课程访问数据到数据库
def save(stat_date, course_id, channel, pv_count, uv_count, ip_count):
    sql = """
            REPLACE INTO `baidu_pv`
                (
                    `stat_date`,
                    `course_id`,
                    `channel`,
                    `pv_count`,
                    `uv_count`,
                    `ip_count`
                )
                VALUES
                (
                 %s,
                 %s,
                 %s,
                 %s,
                 %s,
                 %s)
        """
    params = (stat_date, course_id, channel, pv_count, uv_count, ip_count)
    #print("execute sql=" + sql % params )
    cursor.execute(sql, params)
    lastId = cursor.lastrowid
    return lastId

def encrypt(data):
    # load公钥和密钥
    with open('api_pub.key') as publickfile:
        p = publickfile.read()
        pubkey = rsa.PublicKey.load_pkcs1_openssl_pem(p)

    # 用公钥加密
    n = int(math.ceil(len(data) * 1.0 / 117))
    ret = ''
    for i in range(n):
        gzdata = data[i * 117:(i + 1) * 117]
        ret += rsa.encrypt(gzdata, pubkey)

    # print ret
    return ret


# 解压gzip
def gzdecode(data):
    f = StringIO.StringIO(data)
    gziper = gzip.GzipFile(fileobj=f, compresslevel=9)
    data2 = gziper.read()  # 读取解压缩后数据
    gziper.close()
    return data2


# 压缩gzip
def gzencode(data):
    f = StringIO.StringIO()
    gziper = gzip.GzipFile(fileobj=f, mode='wb', compresslevel=9, )
    gziper.write(data)  # 压缩后数据
    gziper.close()
    return f.getvalue()


class BaiduTongji(object):
    ucid = None
    st = None

    def __init__(self, username, password, token):
        self.username = username
        self.password = password
        self.token = token

        # login
        # self.prelogin()
        ret = self.dologin()
        self.ucid = str(ret['ucid'])
        self.st = ret['st']

    def prelogin(self):
        data = {'username': self.username,
                'token': self.token,
                'functionName': 'preLogin',
                'uuid': UUID,
                'request': {'osVersion': 'windows', 'deviceType': 'pc', 'clientVersion': '1.0'},
                }

        headers = {'UUID': UUID, 'account_type': ACCOUNT_TYPE,
                   'Content-Type': 'data/gzencode and rsa public encrypt;charset=UTF-8'
                   }

        # 压缩
        post_data = gzencode(json.dumps(data))
        # 加密
        post_data = encrypt(post_data)

        resp = requests.post(LOGIN_URL, data=post_data, headers=headers)
        ret = json.loads(gzdecode(resp.content[8:]))
        print 'prelogin:', ret

    def dologin(self):
        data = {'username': self.username,
                'token': self.token,
                'functionName': 'doLogin',
                'uuid': UUID,
                'request': {'password': self.password}
                }

        headers = {'UUID': UUID, 'account_type': ACCOUNT_TYPE,
                   'Content-Type': 'data/gzencode and rsa public encrypt;charset=UTF-8'
                   }

        # 压缩
        post_data = gzencode(json.dumps(data))
        # 加密
        post_data = encrypt(post_data)
        # post
        resp = requests.post(LOGIN_URL, data=post_data, headers=headers)
        ret = json.loads(gzdecode(resp.content[8:]))
        if ret['retcode'] == 0:
            #print u'dologin:', ret['retmsg']
            print u'dologin success'
            print ret['ucid']
            print ret['st']
        return ret

    def dologout(self):
        data = {'username': self.username,
                'token': self.token,
                'functionName': 'doLogout',
                'uuid': UUID,
                'request': {'ucid': self.ucid, 'st': self.st, }
                }

        headers = {'UUID': UUID, 'account_type': ACCOUNT_TYPE,
                   'Content-Type': 'data/gzencode and rsa public encrypt;charset=UTF-8'
                   }

        # 压缩
        post_data = gzencode(json.dumps(data))
        # 加密
        post_data = encrypt(post_data)
        # post
        resp = requests.post(LOGIN_URL, data=post_data, headers=headers)
        ret = json.loads(gzdecode(resp.content[8:]))
        #print 'logout:', ret['retmsg']
        print 'logout: sucess'

    def getsitelist(self):
        url = API_URL + '/getSiteList'
        headers = {'UUID': UUID, 'USERID': self.ucid, 'Content-Type': 'data/json;charset=UTF-8'}
        data = {'header': {'username': self.username, 'password': self.st, 'token': self.token,
                           'account_type': ACCOUNT_TYPE, },
                'body': None, }
        post_data = json.dumps(data)
        resp = requests.post(url, data=post_data, headers=headers)
        # print resp.json()
        return resp.json()['body']['data'][0]['list']

    def getdata(self, para):
        url = API_URL + '/getData'
        headers = {'UUID': UUID, 'USERID': self.ucid, 'Content-Type': 'data/json;charset=UTF-8'}
        data = {'header': {'username': self.username, 'password': self.st, 'token': self.token,
                           'account_type': ACCOUNT_TYPE, },
                'body': para, }

        post_data = json.dumps(data)
        resp = requests.post(url, data=post_data, headers=headers)
        # print resp.json()
        return resp.json()['body']

    def gettoppage(self, stat_date):
        stat_date = stat_date.replace('-','')
        param = {'site_id': SITE_ID,  # 站点ID
            'method': 'visit/toppage/a',  # 趋势分析报告
            'start_date': stat_date,  # 所查询数据的起始日期
            'end_date': stat_date,  # 所查询数据的结束日期
            'metrics': 'pv_count,visitor_count,ip_count',  # 所查询指标为PV和UV
            'max_results': '0',  # 返回所有条数
        }
        ret = self.getdata(param)
        items = ret['data'][0]['result']['items']
        return items

def getChannelFromUrl(urlChannel):
    channelMap = {
        "t": "TED",
        "wx": "WX",
        "other": "OTHER"
    }
    if urlChannel in channelMap:
        return channelMap[urlChannel]
    return "OTHER"


def filterCoursePV(items):
    pages = items[0]
    pvs = items[1]
    coursePV = {}
    for i, _ in enumerate(pages):
        page = pages[i][0]["name"]
        if page.find(HOST) == -1:
            continue
        if page.find("from=") == -1:
            page = page + "&from=other"
        m = re.search('/course/(\d+).*?from=([^%&]*)', page)
        if m:
            courseId = m.group(1)
            urlChannel = m.group(2)
            channel = getChannelFromUrl(urlChannel)
            pv = [0, 0, 0]
            key = courseId + "_" + channel
            if key in coursePV:
                pv = coursePV[key]
            pv[0] += pvs[i][0]
            pv[1] += pvs[i][1]
            pv[2] += pvs[i][2]
            coursePV[key] = pv
        else:
            continue
    return coursePV

if __name__ == '__main__':
    # 配置自己的用户/密码/token
    bdtj = BaiduTongji('test','test','test')
    # 需要统计的日期，默认是最近一周
    stat_dates = []
    if len(sys.argv) == 1:
        today=date.today()
        for days in range(7):
            st = today - timedelta(days=days)
            stat_dates.append(st.strftime('%Y-%m-%d'))
    else:
        for v in sys.argv[1:]:
            if re.match('^20\d{2}-\d{2}-\d{2}$', v):
                stat_dates.append(v)
            else:
                print('unknow date param: ' + str(v))
                sys.exit()
     # 依次同步每一天的统计数据
    for stat_date in stat_dates:
        items = bdtj.gettoppage(stat_date)
        coursePV = filterCoursePV(items)
        for k,v in coursePV.iteritems():
            arr = k.split("_")
            save(stat_date, arr[0], arr[1], v[0], v[1], v[2])
        time.sleep(2)
    bdtj.dologout()
